#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
自动化API收割机 - 增强版
使用Chrome DevTools Protocol捕获网络请求
"""

import time
import json
import requests
import re
from datetime import datetime
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, WebDriverException


class AutoAPIHarvester:
    """自动化API收割机 - 使用CDP捕获请求"""
    
    def __init__(self, headless=True):
        """初始化"""
        self.driver = None
        self.headless = headless
        self.target_url = "https://www.acgice.com/sjz/v/index"
        self.token_cache = {}
        self.api_data_cache = {}
        self.harvest_count = 0
        self.network_requests = []
        
    def setup_browser(self):
        """设置浏览器并启用CDP"""
        print("🚀 正在启动自动化浏览器(启用CDP)...")
        
        chrome_options = Options()
        if self.headless:
            chrome_options.add_argument("--headless")
        
        chrome_options.add_argument("--no-sandbox")
        chrome_options.add_argument("--disable-dev-shm-usage")
        chrome_options.add_argument("--disable-gpu")
        chrome_options.add_argument("--disable-web-security")
        chrome_options.add_argument("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
        chrome_options.add_argument("--window-size=1920,1080")
        
        # 启用性能日志
        chrome_options.set_capability("goog:loggingPrefs", {'performance': 'ALL'})
        
        try:
            # 尝试使用项目根目录的ChromeDriver
            import os
            script_dir = os.path.dirname(os.path.abspath(__file__))
            project_root = os.path.dirname(script_dir)
            chromedriver_path = os.path.join(project_root, "chromedriver.exe")
            
            if os.path.exists(chromedriver_path):
                service = Service(chromedriver_path)
                self.driver = webdriver.Chrome(service=service, options=chrome_options)
                print(f"✅ 使用项目ChromeDriver启动成功")
            else:
                # 尝试系统PATH
                self.driver = webdriver.Chrome(options=chrome_options)
                print("✅ 使用系统ChromeDriver启动成功")
            
            self.driver.set_page_load_timeout(30)
            
            # 启用网络请求捕获
            self.enable_network_monitoring()
            
            return True
            
        except Exception as e:
            print(f"❌ 浏览器启动失败: {e}")
            return False
    
    def enable_network_monitoring(self):
        """启用网络请求监控"""
        print("📡 启用网络请求监控...")
        
        # 获取CDP会话
        self.driver.execute_cdp_cmd('Network.enable', {})
        
        # 设置请求拦截模式（可选）
        self.driver.execute_cdp_cmd('Network.setRequestInterception', {
            'patterns': [{'urlPattern': '*', 'resourceType': 'XHR'}]
        })
        
        # 监听网络请求事件
        self.driver.execute_cdp_cmd('Network.setCacheDisabled', {'cacheDisabled': True})
        
        print("✅ 网络监控已启用")
    
    def visit_and_trigger(self):
        """访问页面并触发API调用"""
        print(f"🌐 正在访问: {self.target_url}")
        
        try:
            # 清空之前的网络请求记录
            self.network_requests = []
            
            # 开始捕获网络请求
            self.driver.get(self.target_url)
            
            # 等待页面加载
            WebDriverWait(self.driver, 15).until(
                EC.presence_of_element_located((By.TAG_NAME, "body"))
            )
            print("✅ 页面加载成功")
            
            # 等待页面完全加载
            time.sleep(3)
            
            # 触发更多API调用
            self.trigger_api_calls()
            
            # 收集网络日志
            self.collect_network_logs()
            
            return True
            
        except Exception as e:
            print(f"❌ 页面访问失败: {e}")
            return False
    
    def trigger_api_calls(self):
        """触发页面的API调用"""
        print("🎯 正在触发API调用...")
        
        try:
            # 刷新页面触发初始API
            self.driver.refresh()
            time.sleep(3)
            
            # 执行JavaScript来触发更多API调用
            trigger_js = """
            // 尝试触发各种API调用
            try {
                // 模拟用户交互
                const buttons = document.querySelectorAll('button, a, [onclick]');
                let clickCount = 0;
                buttons.forEach((btn, index) => {
                    if (index < 5 && btn.offsetParent !== null) { // 只点击前5个可见元素
                        setTimeout(() => {
                            try {
                                btn.click();
                                console.log('点击了元素:', btn);
                            } catch(e) {
                                console.log('点击失败:', e);
                            }
                        }, index * 1000);
                        clickCount++;
                    }
                });
                
                return clickCount;
            } catch(e) {
                console.log('触发API调用时出错:', e);
                return 0;
            }
            """
            
            click_count = self.driver.execute_script(trigger_js)
            print(f"🖱️ 触发了 {click_count} 个元素点击")
            
            # 等待API调用完成
            time.sleep(5)
            
        except Exception as e:
            print(f"⚠️ 触发API调用时出错: {e}")
    
    def collect_network_logs(self):
        """收集网络请求日志"""
        print("📡 收集网络请求日志...")
        
        try:
            # 方法1: 从性能日志中获取网络请求
            logs = self.driver.get_log('performance')
            self.process_performance_logs(logs)
            
            # 方法2: 直接从CDP获取网络请求
            self.process_cdp_network_requests()
            
            # 方法3: 从页面源码中查找API URL
            self.extract_api_urls_from_source()
            
            print(f"📊 总共收集到 {len(self.network_requests)} 个网络请求")
            
        except Exception as e:
            print(f"❌ 收集网络日志失败: {e}")
    
    def process_performance_logs(self, logs):
        """处理性能日志中的网络请求"""
        api_pattern = re.compile(r'/api/sjz/[^?]+\?[^"]*token=[a-f0-9]{32}[^"]*timestamp=\d+')
        
        for log in logs:
            try:
                message = json.loads(log['message'])['message']
                
                if message['method'] == 'Network.requestWillBeSent':
                    url = message['params'].get('request', {}).get('url', '')
                    if '/api/sjz/' in url and api_pattern.search(url):
                        self.process_api_url(url, source='performance_log')
                
                elif message['method'] == 'Network.responseReceived':
                    url = message['params'].get('response', {}).get('url', '')
                    if '/api/sjz/' in url and api_pattern.search(url):
                        self.process_api_url(url, source='performance_log')
            except:
                continue
    
    def process_cdp_network_requests(self):
        """通过CDP获取网络请求"""
        try:
            # 获取所有网络请求
            requests = self.driver.execute_cdp_cmd('Network.getRequestByPattern', {
                'urlPattern': '*api/sjz/*'
            })
            
            if requests and 'requestId' in requests:
                request_id = requests['requestId']
                response = self.driver.execute_cdp_cmd('Network.getResponseBody', {
                    'requestId': request_id
                })
                
                if response:
                    url = requests.get('url', '')
                    self.process_api_url(url, source='cdp')
        except Exception as e:
            print(f"⚠️ 通过CDP获取网络请求时出错: {e}")
    
    def extract_api_urls_from_source(self):
        """从页面源码中提取API URL"""
        try:
            page_source = self.driver.page_source
            api_urls = re.findall(r'https://www\.acgice\.com/api/sjz/[^"\']*\?[^"\']*token=([a-f0-9]{32})[^"\']*timestamp=(\d+)[^"\']*', page_source)
            
            for token, timestamp in api_urls:
                url = f"https://www.acgice.com/api/sjz/?token={token}&timestamp={timestamp}"
                self.process_api_url(url, source='page_source')
        except Exception as e:
            print(f"⚠️ 从页面源码提取API URL时出错: {e}")
    
    def process_api_url(self, url, source='unknown'):
        """处理API URL并提取token和timestamp"""
        try:
            token_match = re.search(r'token=([a-f0-9]{32})', url)
            timestamp_match = re.search(r'timestamp=(\d+)', url)
            endpoint_match = re.search(r'/api/sjz/([^?]+)', url)
            
            if token_match and timestamp_match:
                endpoint = endpoint_match.group(1) if endpoint_match else 'unknown'
                token = token_match.group(1)
                timestamp = timestamp_match.group(1)
                
                # 添加到网络请求列表
                request_info = {
                    'endpoint': endpoint,
                    'token': token,
                    'timestamp': timestamp,
                    'url': url,
                    'source': source,
                    'captured_at': datetime.now().isoformat()
                }
                
                # 避免重复添加
                if not any(req['url'] == url for req in self.network_requests):
                    self.network_requests.append(request_info)
                    print(f"🔍 [{source}] 捕获API: {endpoint} - Token: {token[:8]}... - 时间戳: {timestamp}")
                
                # 更新token缓存
                self.token_cache[endpoint] = {
                    'token': token,
                    'timestamp': timestamp,
                    'url': url,
                    'source': source,
                    'captured_at': datetime.now().isoformat()
                }
                
        except Exception as e:
            print(f"❌ 处理API URL时出错: {e}")
    
    def call_apis_with_tokens(self):
        """使用收集到的token调用API"""
        if not self.token_cache:
            print("❌ 没有可用的token")
            return
        
        print(f"\n🚀 开始调用 {len(self.token_cache)} 个API...")
        
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Referer': 'https://www.acgice.com/sjz/v/index',
            'Origin': 'https://www.acgice.com',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin'
        }
        
        for endpoint, token_info in self.token_cache.items():
            try:
                api_url = token_info['url']
                
                print(f"\n📡 调用API: {endpoint}")
                print(f"🔗 URL: {api_url}")
                
                response = requests.get(api_url, headers=headers, timeout=10)
                
                if response.status_code == 200:
                    try:
                        data = response.json()
                        self.api_data_cache[endpoint] = {
                            'data': data,
                            'fetched_at': datetime.now().isoformat(),
                            'token_used': token_info['token'][:8] + '...',
                            'status': 'success'
                        }
                        
                        print(f"✅ {endpoint} 调用成功!")
                        print(f"📊 数据大小: {len(json.dumps(data))} 字符")
                        
                        # 显示数据预览
                        if isinstance(data, dict):
                            if 'data' in data:
                                if isinstance(data['data'], list):
                                    print(f"📋 数据项数: {len(data['data'])}")
                                elif isinstance(data['data'], dict):
                                    print(f"📋 数据字段: {list(data['data'].keys())}")
                            print(f"📄 响应预览: {json.dumps(data, ensure_ascii=False)[:200]}...")
                        
                    except json.JSONDecodeError:
                        print(f"❌ {endpoint} 响应不是JSON格式")
                        self.api_data_cache[endpoint] = {
                            'data': response.text,
                            'fetched_at': datetime.now().isoformat(),
                            'status': 'invalid_json'
                        }
                else:
                    print(f"❌ {endpoint} 调用失败: HTTP {response.status_code}")
                    print(f"📄 错误内容: {response.text[:200]}...")
                    
            except Exception as e:
                print(f"❌ {endpoint} 请求异常: {e}")
    
    def output_results(self):
        """输出所有结果"""
        print("\n" + "=" * 80)
        print("📋 API数据收割结果汇总")
        print("=" * 80)
        
        if not self.api_data_cache:
            print("❌ 没有成功获取到任何API数据")
            return
        
        for endpoint, result in self.api_data_cache.items():
            print(f"\n🔗 API端点: {endpoint}")
            print(f"📅 获取时间: {result['fetched_at']}")
            print(f"🔑 使用Token: {result.get('token_used', 'N/A')}")
            print(f"📊 状态: {result['status']}")
            
            if result['status'] == 'success':
                print("📋 完整数据:")
                print("-" * 60)
                print(json.dumps(result['data'], indent=2, ensure_ascii=False))
                print("-" * 60)
            else:
                print(f"❌ 数据获取失败: {result.get('data', 'Unknown error')}")
        
        # 保存到文件
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"api_harvest_{timestamp}.json"
        
        try:
            with open(filename, 'w', encoding='utf-8') as f:
                json.dump({
                    'metadata': {
                        'harvest_time': timestamp,
                        'target_url': self.target_url,
                        'total_apis': len(self.api_data_cache)
                    },
                    'network_requests': self.network_requests,
                    'api_data': self.api_data_cache
                }, f, indent=2, ensure_ascii=False)
            print(f"\n💾 结果已保存到: {filename}")
        except Exception as e:
            print(f"\n❌ 保存文件失败: {e}")
    
    def cleanup(self):
        """清理资源"""
        if self.driver:
            print("\n🧹 正在清理浏览器资源...")
            self.driver.quit()
            print("✅ 清理完成")
    
    def run_harvest(self):
        """运行完整的收割流程"""
        try:
            print("🌾 启动自动化API收割机(CDP版)")
            print("=" * 60)

            # 1. 启动浏览器并启用CDP
            if not self.setup_browser():
                return False

            # 2. 访问页面并捕获网络请求
            if not self.visit_and_trigger():
                return False

            # 3. 如果没有收集到token，再尝试几次
            retry_count = 0
            while not self.token_cache and retry_count < 3:
                retry_count += 1
                print(f"🔄 第 {retry_count} 次重试收集token...")
                
                # 再次触发API调用
                self.trigger_api_calls()
                time.sleep(5)
                
                # 重新收集网络日志
                self.collect_network_logs()

            if self.token_cache:
                # 4. 使用token调用API
                self.call_apis_with_tokens()

                # 5. 输出结果
                self.output_results()

                self.harvest_count += 1
                print(f"\n🎉 第 {self.harvest_count} 次收割完成!")
                return True
            else:
                print("❌ 多次尝试后仍无法收集到token")
                print("💡 可能的原因:")
                print("   1. 页面结构已更新，API路径已改变")
                print("   2. 网站使用了新的反爬虫机制")
                print("   3. 需要用户登录或特定操作才能触发API")
                return False

        finally:
            self.cleanup()


def main():
    """主函数"""
    print("🌾 自动化API收割机(CDP增强版)")
    print("使用Chrome DevTools Protocol捕获网络请求")
    print("自动访问网页 -> 捕获API请求 -> 调用API -> 输出数据")
    print("")

    # 创建收割机实例
    harvester = AutoAPIHarvester(headless=False)
    
    # 运行收割
    success = harvester.run_harvest()
    
    if success:
        print("\n✅ 自动化收割成功完成!")
    else:
        print("\n❌ 自动化收割失败")
        print("💡 建议:")
        print("   1. 检查网络连接")
        print("   2. 确保ChromeDriver版本与浏览器匹配")
        print("   3. 尝试显示浏览器窗口进行调试")


if __name__ == "__main__":
    main()